#!/usr/bin/python

# Copyright 2009, Fei Yu, feiyu [at] berkeley [dot] edu

# This file is part of Topics-Portal.
#
# Topics-Portal is free software: you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Topics-Portal is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with PyGrams.  If not, see <http://www.gnu.org/licenses/>.

from django.shortcuts import render_to_response, get_object_or_404
from django.http import Http404, HttpResponseRedirect
from django.core.urlresolvers import reverse
from django.conf import settings
# from ims.sti.models import *
from couchdbkit import *
import re
import copy
import difflib
import simplejson
import urllib
import urllib2

SEARCH_LIMIT = 100

server = client.Server(settings.COUCHDB_HOST)
db = server[settings.COUCHDB_DATABASE_NAME]

url_head = settings.COUCHDB_HOST + settings.COUCHDB_DATABASE_NAME
view_doc = {'all': url_head + "/_fti/lucene2/all",
			'allFields': url_head + "/_fti/lucene2/allf",
			'allDocs': url_head + "/_all_docs",
			'bookIndex': url_head + "/_design/lucene2/_view/bookIndex",
			'topicIndex': url_head + "/_design/lucene2/_view/topicIndex",
			'courseIndex': url_head + "/_design/lucene2/_view/courseIndex",
			'topicRefInBooks': url_head + "/_design/lucene2/_view/topicRefInBooks",
			'topicRefInLectures': url_head + "/_design/lucene2/_view/topicRefInLectures",
	}
	
	
def unicode_x_to_u00(s):
	"""convert unicode in formate \xEE to \u00EE.
		This conversion is necessary for lucene queries to 
		work properly with non-ascii characters.
	"""
	if type(s) == type(unicode('a')):
		s1 = repr(s)
		s1 = s1[2:-1]	# format: u'...'
		s1 = s1.replace('\\x', '\\u00')
		return unicode(s1)
	return s

def bulkGetUniqueTopics(all_topic_ids):
	"""A helper function. Given a list of topic ids which are not necessarily unique,
	the function will return a dict object that maps a topic id to its corresponding
	value in the topicIndex view."""
	unique_topic_ids = list(set(all_topic_ids))
	unique_topic_docs = simplejson.load(urllib2.urlopen(view_doc['topicIndex'],
						simplejson.dumps({"keys": unique_topic_ids})))['rows']
	topics = dict([(row['id'], row['value']) for row in unique_topic_docs])
	return topics

def bulkGetUniqueDocs(all_doc_ids):
	"""A helper function. Given a list of doc ids which are not necessarily unique,
	the function will return a dict object that maps a topic id to its corresponding
	doc."""
	unique_doc_ids = list(set(all_doc_ids))
	unique_doc_docs = simplejson.load(urllib2.urlopen(\
					view_doc['allDocs']+'?include_docs=true',
						simplejson.dumps({"keys": unique_doc_ids})))['rows']
	docs = dict([(row['id'], row['doc']) for row in unique_doc_docs])
	return docs

def home(request):
	return render_to_response('home.html', {'title': 'Welcome'})


def topic(request, topicName):
	def getBookRef(topic):
		"""Find all occurences of the topic in all the books"""
		topic_id = topic['_id']
		all_ref = simplejson.load(urllib.urlopen(\
					view_doc['topicRefInBooks'] + '?' + urllib.urlencode(\
					{"key": simplejson.dumps(topic_id)})))['rows']
		book_ids = list(set([row['value']['bookID'] for row in all_ref]))
		rows = simplejson.load(urllib2.urlopen(view_doc['bookIndex'],
					simplejson.dumps({"keys": book_ids})))['rows']
		books = [row['value'] for row in rows]
		book_dict = dict([(b, []) for b in book_ids])
		for row in all_ref:
			page = row['value']['page']
			if page:
				book_dict[row['value']['bookID']].append(page)
		references = [(b, book_dict[b['id']]) for b in books]
		return sorted(references, 
						cmp=lambda x,y: cmp(x[0]['id'], y[0]['id']))
	
	def getLectureRef(topic):
		"""Find all occurences of the topic in all the courses"""
		topic_id = topic['_id']
		all_ref = simplejson.load(urllib.urlopen(\
					view_doc['topicRefInLectures'] + '?' + \
					urllib.urlencode({"key": simplejson.dumps(topic_id)})))['rows']	
		course_ids = list(set([row['value']['courseID'] for row in all_ref]))
		rows = simplejson.load(urllib2.urlopen(view_doc['courseIndex'],
					simplejson.dumps({"keys": course_ids})))['rows']
		courses = [row['value'] for row in rows]
		course_dict = dict([(c, []) for c in course_ids])
		for row in all_ref:
			course_dict[row['value']['courseID']].append(row['value'])
		references = [(c, course_dict[c['id']]) for c in courses]
		return sorted(references, 
						cmp=lambda x,y: cmp(x[0]['id'], y[0]['id']))
	
	def getChildren(topic):
		"""get child topics of topic"""
		all_topic_ids = []
		children={}
		for cat in topic['categories']:
			assoc_topics = set([a['target'] for a in cat['associations'] if \
								a['type'] == 'parent to child'])
			if len(assoc_topics) > 0:
				all_topic_ids += assoc_topics
				children[cat['category']] = assoc_topics
		results = {}
		topic_dict = bulkGetUniqueTopics(all_topic_ids)
		for cat in children:
			results[cat] = [topic_dict[t] for t in children[cat]]
			results[cat] = sorted(results[cat], \
								lambda x, y: cmp(x['id'], y['id']))
		return results

	def getParents(topic):
		"""get parent topics of topic"""
		url = view_doc['allFields'] + '?include_docs=true&' + \
				'q=doc_type:Topic+AND+target:' + unicode_x_to_u00(topic['_id'])
		all_parents_rows = simplejson.load(urllib.urlopen(url))['rows']
		all_parents = [row['doc'] for row in all_parents_rows]
		for doc in all_parents:
			doc['id'] = doc['_id']
		
		parents={}
		for index,t in enumerate(all_parents):
			if t['_id'] == topic['_id']:
				continue
			for cat in t['categories']:
				category = cat['category']
				if category not in parents:
					parents[category] = set([])
				for assoc in cat['associations']:
					if topic['_id'] == assoc['target'] and \
					   assoc['type'] == "parent to child":
						parents[category] |= set([index])
		results = {}
		for cat in parents:
			results[cat] = [all_parents[index] for index in \
							sorted(parents[cat])]
			results[cat] = sorted(results[cat], \
								lambda x, y: cmp(x['_id'], y['_id']))
		return results	
	
	def getRelated(topic):
		"""Get related/sibling topics of topic"""
		result = set([])
		stack = set([(topic['_id'], None)])
		while len(stack) > 0:
			topic_id,category = stack.pop()
			result |= set([(topic_id, category)])
			topic = db.view('lucene2/topicIndex', \
							key=topic_id, include_docs='true').first()
			if not topic:
				continue
			else:
				topic = topic['doc']
			for cat in topic['categories']:
				for assoc in cat['associations']:
					if assoc['type']=='related':
						stack |= set([(assoc['target'], \
										cat['category'])]).difference(result)
		related={}
		for t in result:
			if t[0] == topic['_id']:
				continue
			if t[1] not in related:
				related[t[1]] = set([])
			related[t[1]] |= set([t[0]])
		all_topic_ids = []
		for cat in related:
			all_topic_ids.append(related[cat])
		topic_dict = bulkGetUniqueTopics(all_topic_ids)
		
		results = {}
		for cat in related:
			results[cat] = [topic_dict[t] for t in related[cat]]
			results[cat] = sorted(results[cat], \
								lambda x, y: cmp(x['id'], y['id']))
		return results

	def getPractioners(data):
		"""Extract from data documents of type "Person" """
		return [doc for doc in data if doc['doc_type'] == 'Person']
	
	def getSource(topic, source_type):
		"""Get the topic's url of source_type, where source_type 
		is chosen from Wikipedia, Mathworld, PlanetMath and ISI
		"""
		results={}
		for cat in topic['categories']:
			category = cat['category']
			if not category in results:
				results[category] = []
			for source in cat['sources']:
				if source['type'] == source_type:
					results[category].append(source)
		return results
			

		
	try:
		topic=db.get("topics/" + topicName)
	except client.ResourceNotFound:
		raise Http404


	source_choices = ['MathWorld', 'PlanetMath', 'Wikipedia', 'ISI']
	
	children = getChildren(topic)
	parents = getParents(topic)
	related	= getRelated(topic)
	isi_url = getSource(topic,source_choices[3])
	aux_encyclopedia_urls = [getSource(topic, source_type) for \
							source_type in source_choices[:3]]
	
	# Collect all the data obtained above
	data = {}
	for category in children:
		if not category in data:
			data[category] = {'name': category,}
		data[category]['children'] = children[category]
	for category in parents:
		if not category in data:
			data[category] = {'name': category,}
		data[category]['parents'] = parents[category] 
	for category in related:
		if not category in data:
			data[category] = {'name': category,}
		data[category]['related'] = related[category]
	for category in isi_url:
		if not category in data:
			data[category] = {'name': category,}
		data[category]['isi_url'] = isi_url[category]
	
	for url_type in aux_encyclopedia_urls:
		for category in url_type:
			if not category in data:
				data[category] = {'name': category,}
			if not 'aux_encyclopedia_urls' in data[category]:
				data[category]['aux_encyclopedia_urls'] = []
			data[category]['aux_encyclopedia_urls'] += url_type[category]
	
	context = {
		'topic': topic,
		'title': topic['name'],
		'data': [data[category] for category in data],
		'references_book': getBookRef(topic),
		'references_lecture': getLectureRef(topic),		
		}
	return render_to_response('topic.html', context)


def topicIndex(request):
	"""List topics that start with the alphabet requested"""
	page = request.GET.get('page', 'A')
	start = page.lower()
	end = chr(ord(start) + 1) if chr(ord(start) + 1).isalpha() else None
	db_query = ['startkey="topics/%s"' % start]
	if end: 
		db_query.append('endkey="topics/%s"' % end)
	
	url = view_doc['topicIndex'] + '?' + '&'.join(db_query)
	topics = [row['value'] for row in \
					simplejson.load(urllib.urlopen(url))['rows']]
	context = {
		"topics": topics,
		"current_letter": page,
		"title": "Topic Index"
		}
	return render_to_response('topicIndex.html', context)

def search(request):
	query = request.GET['q'].lower()
	
	exactMatch = db.view('lucene2/topicIndex', key='topics/' + query).first()
	if not exactMatch:
		exactMatch = db.view('lucene2/topicIndex', \
							key='topics/' + query.replace(' ', '_')).first()
	if exactMatch:
		return HttpResponseRedirect('/' + exactMatch['id'])
	
	## 
	url = view_doc['allFields'] + \
		'?include_docs=true&limit=5&q=doc_type:Topic+AND+' + \
		'_id:topics/%s~0.7' % \
		unicode_x_to_u00(query.replace(' ','_')) #approximate search in lucene
	approxMatches = [row['doc'] for row in \
						simplejson.load(urllib.urlopen(url))['rows']]
						
	url = view_doc['allFields'] + \
			'?q=doc_type:Topic+AND+_id:topics/%s&include_docs=true' % \
			unicode_x_to_u00(query.replace(' ','_'))
	incompleteMatches = [row['doc'] for row in \
							simplejson.load(urllib.urlopen(url))['rows']]
	uniqueTopics = {}
	for doc in approxMatches + incompleteMatches:
		if not doc['_id'] in uniqueTopics:
			uniqueTopics[doc['_id']] = doc
	matches = uniqueTopics.values()
	if matches:
		for doc in matches:
			doc['id'] = doc['_id']
		context = {'matches':sorted(matches,\
					lambda x, y:cmp(x['id'], y['id']))}
		return render_to_response('searchResults.html', context)
	
	return render_to_response('noResults.html')

def personIndex(request):
	"""List all the people and topics that they know"""
	person_docs = simplejson.load(urllib.urlopen(view_doc['allFields'] + \
					"?include_docs=true&limit=100&q=doc_type:Person"))['rows']
	persons = [row['doc'] for row in person_docs]
	all_topic_ids = []
	# get all topics at once
	for person in persons:
		if 'knowledge' in person:
			all_topic_ids += [k['topic'] for k in person['knowledge']]
	unique_topics = bulkGetUniqueTopics(all_topic_ids)	
	
	data = []
	for person in persons:
		if not 'knowledge' in person:	
			continue
		topics = [unique_topics[k['topic']] for k in person['knowledge']]
		topics = sorted(topics, lambda x, y: cmp(x['id'], y['id']))
		data.append((person['first_name'], person['last_name'], topics))
	context = {'title': 'Person Index',
				'data': sorted(data, lambda x, y: cmp(x[1], y[1]))
				}
	return render_to_response('personIndex.html', context)


def courseIndex(request):
	courses = [row['value'] for row in db.view('lucene2/courseIndex').all()]
	context = {
		"courses": courses,
		"title": "Course Index"
		}
	return render_to_response('courseIndex.html', context)


def course(request, courseName, choice='lecture'):
	"""
	Key arguments:
	courseName -- the courseName is in doc_id format
	choice -- signify whether to return book data or lecture data
				(default is lecture data)
	"""
	try:
		course = db.get("courses/"+courseName)
	except ResourceNotFound:
		raise Http404
	
	def availability(course):
		"""check whether book data or lecture data are available"""
		l_flag = False
		b_flag = False
		if course['lectures']: 
			l_flag = True
		if course['books']: 
			b_flag = True
		
		if l_flag and b_flag: 
			return 'both'
		elif l_flag: 
			return 'lecture'
		elif b_flag: 
			return 'book'
		else: 
			return 'none'
	
	semester = "[%s]" % course['semester'] if course['semester'] else ""
	name = course['name'] if course['name'] else ""
	description = "-- " + course['description'] if \
					course['description'] else ""
	
	course = copy.deepcopy(course)
	course['id'] = course['_id']
	
	if choice == 'lecture':
		all_topic_ids = []
		for lecture in course['lectures']:
			all_topic_ids += lecture['topics']
		unique_topics = bulkGetUniqueTopics(all_topic_ids)	
		
		for lecture in course['lectures']:
			lecture['topics'] = [unique_topics[topID] for topID in lecture['topics']]
		
		books = [db.view('lucene2/bookIndex',key=bk).first() for \
					bk in course['books']]
		books = [bk['value'] for bk in books if bk]
		book_data = books
	else:		
		books = [db.view('lucene2/bookIndex', \
				key=bk, include_docs='true').first() for \
				bk in course['books']]
		books = [bk['doc'] for bk in books if bk]
		book_data = [generate_book_data(bk) for bk in books]


		
	context = {
		'choice' : choice,
		'availability' : availability(course),
		'course' : course,
		'title' : ' '.join([semester, name, description]),
		'books' : book_data
		}
	return render_to_response('course.html', context)


def bookIndex(request):
	books = [row['value'] for row in \
			simplejson.load(urllib.urlopen(view_doc['bookIndex']))['rows']]
	all_author_ids = []
	all_publisher_ids = []
	for bk in books:
		all_author_ids += bk['authors']
		all_publisher_ids += [bk['publisher'],]
	unique_authors = bulkGetUniqueDocs(all_author_ids)
	unique_publishers = bulkGetUniqueDocs(all_publisher_ids)

	data = []	
	for bk in books:
		authors = [unique_authors[a] for a in bk['authors']]
		authors = [' '.join([a['first_name'] ,a['last_name']]) for \
					a in authors]
		publisher = unique_publishers[bk['publisher']]
		publisher = publisher['name'] if publisher else None
		data.append((bk, authors, publisher))
		
	context = {
		"data": data,
		"title": "Book Index",
		}
	return render_to_response('bookIndex.html', context)

def generate_book_data(book):
	def getBookIDs(head, container):
		container += head['topics']
		for c in head['children']:
			getBookIDs(c, container)
	
	def setBookTopics(head, unique_topics):
		head['topics'] = [unique_topics[topID] for topID in head['topics']]
		for c in head['children']:
			setBookTopics(c, unique_topics)
		
	book = copy.deepcopy(book)
	all_topic_ids = []
	for chap in book['toc']:
		getBookIDs(chap, all_topic_ids)
	unique_topics = bulkGetUniqueTopics(all_topic_ids)	
	for chap in book['toc']:		
		setBookTopics(chap, unique_topics)
	
	book['authors'] = simplejson.load(urllib2.urlopen( \
						view_doc['allDocs'] + '?include_docs=true',
						simplejson.dumps({"keys": book['authors']})))['rows']
	book['authors'] = [row['doc'] for row in book['authors']]
	
	book['publisher'] = simplejson.load(urllib2.urlopen( \
						view_doc['allDocs'] + '?include_docs=true',
						simplejson.dumps({"keys": [book['publisher'], ]})))['rows']
	book['publisher'] = book['publisher'][0] if book['publisher'] else None
	return book

def book(request, bookName):
	try:
		book = db.get('books/' + bookName)
	except ResourceNotFound:
		raise Http404
	
	url = view_doc['allFields'] + '?include_docs=true' +\
			'&q=books:%s+AND+doc_type:Course' % unicode_x_to_u00(book['_id'])
	rows = simplejson.load(urllib.urlopen(url))['rows']
	all_courses = [row['doc'] for row in rows]
	for c in all_courses:
		c['id'] = c['_id']

	context = {
		'book' : generate_book_data(book),
		'title': book['title'],
		'courses' : [c for c in all_courses if book['_id'] in c['books']]
		}
	return render_to_response('book.html', context) 

def redirect(request,ext_url):
	return HttpResponseRedirect(ext_url)

def about(request):
	return render_to_response('about.html', {'title': 'About'})

